Overview of tutorial
This tutorial simulates a population effect size of Cohen’s d = 0.5
for different sample sizes, and examines the relationship between
Cohen’s d, its 95% Confidence Interval, and the significance of the
t-test’s p-value.
By the end of this lesson you should understand that
p-values are re-expressions of the same information conveyed by
Confidence Intervals, and that statistical power is a re-expression of
the width of Confidence Intervals.
Dependencies
library(tidyr)
library(dplyr)
library(purrr)
library(stringr)
library(forcats)
library(ggplot2)
library(scales)
library(patchwork)
library(knitr)
library(kableExtra)
library(janitor)
library(effsize)
Simulation
# functions for simulation
generate_data <- function(n_per_condition,
mean_control,
mean_intervention,
sd) {
data_control <-
tibble(condition = "control",
score = rnorm(n = n_per_condition, mean = mean_control, sd = sd))
data_intervention <-
tibble(condition = "intervention",
score = rnorm(n = n_per_condition, mean = mean_intervention, sd = sd))
data_combined <- bind_rows(data_control,
data_intervention) |>
mutate(condition = fct_relevel(condition, "intervention", "control"))
return(data_combined)
}
analyze <- function(data) {
res_t_test <- t.test(formula = score ~ condition,
data = data,
var.equal = TRUE,
alternative = "two.sided")
res_cohens_d <- effsize::cohen.d(formula = score ~ condition,
data = data,
pooled = TRUE)
res <- tibble(p = res_t_test$p.value,
cohens_d = res_cohens_d$estimate,
cohens_d_ci_lower = res_cohens_d$conf.int[1],
cohens_d_ci_upper = res_cohens_d$conf.int[2])
return(res)
}
# set seed
set.seed(42)
# simulation parameters
experiment_parameters <- expand_grid(
n_per_condition = seq(from = 10, to = 90, by = 10),
mean_control = 0,
mean_intervention = 0.5,
sd = 1,
iteration = 1:1000
)
# run simulation
simulation <- experiment_parameters |>
mutate(generated_data = pmap(list(n_per_condition,
mean_control,
mean_intervention,
sd),
generate_data)) |>
mutate(results = pmap(list(generated_data),
analyze))
Cohen’s d by sample size
# wrangle
simulation |>
unnest(results) |>
# plot
ggplot(aes(n_per_condition*2, cohens_d)) +
geom_jitter(alpha = 0.25) +
geom_hline(yintercept = 0, linetype = "dotted") +
scale_x_continuous(breaks = breaks_pretty(n = 10),
name = "Total N") +
scale_y_continuous(breaks = breaks_pretty(n = 10),
#limits = c(0,1),
name = "Cohen's d") +
theme_linedraw() +
scale_color_viridis_d(begin = 0.3, end = 0.7) +
ggtitle("Population Cohen's d = 0.5")

Ordered by statistical significance
Let’s color the Cohen’s ds by their statistical significance.
# wrangle
simulation |>
unnest(results) |>
mutate(significant = p < .05) |>
# plot
ggplot(aes(n_per_condition*2, cohens_d, color = significant)) +
geom_jitter(alpha = 0.25) +
geom_hline(yintercept = 0, linetype = "dotted") +
scale_x_continuous(breaks = breaks_pretty(n = 10),
name = "Total N") +
scale_y_continuous(breaks = breaks_pretty(n = 10),
#limits = c(0,1),
name = "Cohen's d") +
theme_linedraw() +
scale_color_viridis_d(begin = 0.3, end = 0.7) +
ggtitle("Population Cohen's d = 0.5")

Cohen’s d and its 95% CIs
Let’s add the 95% Confidence Intervals.
Randomly select one dataset per sample size
# wrangle
simulation |>
unnest(results) |>
mutate(significant = p < .05) |>
group_by(n_per_condition) |>
slice_sample(n = 1) |>
ungroup() |>
# plot
ggplot(aes(n_per_condition*2, cohens_d, color = significant)) +
geom_point() +
geom_linerange(aes(ymin = cohens_d_ci_lower, ymax = cohens_d_ci_upper)) +
geom_hline(yintercept = 0, linetype = "dotted") +
scale_x_continuous(breaks = breaks_pretty(n = 10),
name = "Total N") +
scale_y_continuous(breaks = breaks_pretty(n = 10),
#limits = c(0,1),
name = "Cohen's d") +
theme_linedraw() +
scale_color_viridis_d(begin = 0.3, end = 0.7) +
ggtitle("Population Cohen's d = 0.5\nOne randomly selected dataset per sample size")

All datasets
# wrangle
simulation |>
unnest(results) |>
mutate(significant = p < .05,
total_n = paste("N =", n_per_condition*2),
total_n = fct_reorder(total_n, as.numeric(str_extract(total_n, "\\d+")))) |>
# plot
ggplot(aes(iteration, cohens_d, color = significant)) +
geom_point(alpha = 0.5) +
geom_linerange(aes(ymin = cohens_d_ci_lower, ymax = cohens_d_ci_upper), alpha = 0.2) +
geom_hline(yintercept = 0, linetype = "dotted") +
scale_x_continuous(breaks = breaks_pretty(n = 10),
name = "Iteration") +
scale_y_continuous(breaks = breaks_pretty(n = 10),
#limits = c(0,1),
name = "Cohen's d") +
theme_linedraw() +
scale_color_viridis_d(begin = 0.3, end = 0.7) +
facet_wrap(~ total_n, ncol = 3, scales = "free_y")

Ordered by Cohen’s d
The above plot is hard to understand. Let’s order the Cohen’s ds from
smallest to largest.
# wrangle
simulation |>
unnest(results) |>
mutate(significant = p < .05,
total_n = paste("N =", n_per_condition*2),
total_n = fct_reorder(total_n, as.numeric(str_extract(total_n, "\\d+")))) |>
arrange(n_per_condition, cohens_d) |>
group_by(n_per_condition) |>
mutate(rank = row_number()) |>
ungroup() |>
# plot
ggplot(aes(rank, cohens_d, color = significant)) +
geom_point() +
geom_linerange(aes(ymin = cohens_d_ci_lower, ymax = cohens_d_ci_upper), alpha = 0.2) +
geom_hline(yintercept = 0, linetype = "dotted") +
scale_x_continuous(breaks = breaks_pretty(n = 10),
name = "Ranked iteration") +
scale_y_continuous(breaks = breaks_pretty(n = 10),
#limits = c(0,1),
name = "Cohen's d") +
theme_linedraw() +
scale_color_viridis_d(begin = 0.3, end = 0.7) +
facet_wrap(~ total_n, ncol = 3)

- Notice the relationship between 95% CI and p value
significance.
- A certain percentage of Cohen’s ds are green vs. blue. What is this
percentage also called? What statistical property?
Mean Cohen’s d by sample size
Now let’s average over the Cohen’s ds in each condition to find the
mean Cohen’s d and its mean 95% CIs.
# wrangle
simulation_summary <- simulation |>
# unnest results
unnest(results) |>
group_by(n_per_condition) |>
summarize(proportion_significant = mean(p < .05),
mean_cohens_d = mean(cohens_d),
mean_cohens_d_ci_lower = mean(cohens_d_ci_lower),
mean_cohens_d_ci_upper = mean(cohens_d_ci_upper)) |>
mutate(centered_mean_cohens_d_ci_lower = mean_cohens_d_ci_lower - mean_cohens_d,
centered_mean_cohens_d_ci_upper = mean_cohens_d_ci_upper - mean_cohens_d)
# plot results
p1 <- ggplot(simulation_summary, aes(n_per_condition*2, mean_cohens_d)) +
geom_point() +
geom_linerange(aes(ymin = mean_cohens_d_ci_lower, ymax = mean_cohens_d_ci_upper)) +
geom_hline(yintercept = 0, linetype = "dotted") +
scale_x_continuous(breaks = breaks_pretty(n = 10),
name = "Total sample size") +
scale_y_continuous(breaks = breaks_pretty(n = 5),
#limits = c(0,1),
name = "Mean Cohen's d\n(and mean 95% CIs)") +
theme_linedraw() +
ggtitle("Population Cohen's d = 0.5")
p1

Mean Cohen’s d and power by sample size
p2 <- ggplot(simulation_summary, aes(n_per_condition*2, proportion_significant)) +
geom_point() +
geom_hline(yintercept = 0.05, linetype = "dotted") +
geom_hline(yintercept = 0.80, linetype = "dotted") +
scale_x_continuous(breaks = breaks_pretty(n = 10),
name = "Total sample size") +
scale_y_continuous(breaks = breaks_pretty(n = 5),
limits = c(0,1),
name = "Proportion of significant\np-values") +
theme_linedraw()
p1 + p2 + plot_layout(ncol = 1)

Session info
## R version 4.3.3 (2024-02-29)
## Platform: aarch64-apple-darwin20 (64-bit)
## Running under: macOS 15.2
##
## Matrix products: default
## BLAS: /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/lib/libRblas.0.dylib
## LAPACK: /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/lib/libRlapack.dylib; LAPACK version 3.11.0
##
## locale:
## [1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8
##
## time zone: Europe/Zurich
## tzcode source: internal
##
## attached base packages:
## [1] stats graphics grDevices utils datasets methods base
##
## other attached packages:
## [1] effsize_0.8.1 janitor_2.2.1 kableExtra_1.4.0
## [4] knitr_1.49 patchwork_1.2.0.9000 scales_1.3.0
## [7] ggplot2_3.5.1 forcats_1.0.0 stringr_1.5.1
## [10] purrr_1.0.4 dplyr_1.1.4 tidyr_1.3.1
##
## loaded via a namespace (and not attached):
## [1] gtable_0.3.6 jsonlite_1.8.9 compiler_4.3.3 tidyselect_1.2.1
## [5] xml2_1.3.6 snakecase_0.11.1 jquerylib_0.1.4 systemfonts_1.0.6
## [9] yaml_2.3.10 fastmap_1.2.0 R6_2.6.1 generics_0.1.3
## [13] tibble_3.2.1 munsell_0.5.1 lubridate_1.9.4 svglite_2.1.3
## [17] bslib_0.8.0 pillar_1.10.1 rlang_1.1.5 cachem_1.1.0
## [21] stringi_1.8.4 xfun_0.49 sass_0.4.9 timechange_0.3.0
## [25] viridisLite_0.4.2 cli_3.6.4 withr_3.0.2 magrittr_2.0.3
## [29] digest_0.6.37 grid_4.3.3 rstudioapi_0.17.1 lifecycle_1.0.4
## [33] vctrs_0.6.5 evaluate_1.0.1 glue_1.8.0 farver_2.1.2
## [37] colorspace_2.1-1 rmarkdown_2.29 tools_4.3.3 pkgconfig_2.0.3
## [41] htmltools_0.5.8.1
LS0tCnRpdGxlOiAiVW5kZXJzdGFuZGluZyB0aGUgbGluayBiZXR3ZWVuIHAtdmFsdWVzLCBDb25maWRlbmNlIEludGVydmFscywgYW5kIHBvd2VyIgphdXRob3I6ICJJYW4gSHVzc2V5IgpkYXRlOiAiYHIgZm9ybWF0KFN5cy50aW1lKCksICclZCAlQiwgJVknKWAiCm91dHB1dDoKICBodG1sX2RvY3VtZW50OgogICAgY29kZV9kb3dubG9hZDogdHJ1ZQogICAgY29kZV9mb2xkaW5nOiBzaG93CiAgICBoaWdobGlnaHQ6IGhhZGRvY2sKICAgIHRoZW1lOiBmbGF0bHkKICAgIHRvYzogeWVzCiAgICB0b2NfZmxvYXQ6IHllcwotLS0KCiMgT3ZlcnZpZXcgb2YgdHV0b3JpYWwKClRoaXMgdHV0b3JpYWwgc2ltdWxhdGVzIGEgcG9wdWxhdGlvbiBlZmZlY3Qgc2l6ZSBvZiBDb2hlbidzIGQgPSAwLjUgZm9yIGRpZmZlcmVudCBzYW1wbGUgc2l6ZXMsIGFuZCBleGFtaW5lcyB0aGUgcmVsYXRpb25zaGlwIGJldHdlZW4gQ29oZW4ncyBkLCBpdHMgOTUlIENvbmZpZGVuY2UgSW50ZXJ2YWwsIGFuZCB0aGUgc2lnbmlmaWNhbmNlIG9mIHRoZSB0LXRlc3QncyAqcCotdmFsdWUuIAoKQnkgdGhlIGVuZCBvZiB0aGlzIGxlc3NvbiB5b3Ugc2hvdWxkIHVuZGVyc3RhbmQgdGhhdCAqcCotdmFsdWVzIGFyZSByZS1leHByZXNzaW9ucyBvZiB0aGUgc2FtZSBpbmZvcm1hdGlvbiBjb252ZXllZCBieSBDb25maWRlbmNlIEludGVydmFscywgYW5kIHRoYXQgc3RhdGlzdGljYWwgcG93ZXIgaXMgYSByZS1leHByZXNzaW9uIG9mIHRoZSB3aWR0aCBvZiBDb25maWRlbmNlIEludGVydmFscy4KCiMgQ2l0YXRpb24gJiBMaWNlbnNlCgpDaXRhdGlvbjogCgpJYW4gSHVzc2V5ICgyMDI0KSBJbXByb3ZpbmcgeW91ciBzdGF0aXN0aWNhbCBpbmZlcmVuY2VzIHRocm91Z2ggc2ltdWxhdGlvbiBzdHVkaWVzIGluIFIuIGh0dHBzOi8vZ2l0aHViLmNvbS9pYW5odXNzZXkvc2ltdWxhdGlvbi1jb3Vyc2UKCkxpY2Vuc2U6IAoKW0NDIEJZIDQuMF0oaHR0cHM6Ly9jcmVhdGl2ZWNvbW1vbnMub3JnL2xpY2Vuc2VzL2J5LzQuMC9kZWVkLmVuKQoKYGBge3IsIGluY2x1ZGU9RkFMU0V9CgojIHNldCBkZWZhdWx0IGNodW5rIG9wdGlvbnMKa25pdHI6Om9wdHNfY2h1bmskc2V0KG1lc3NhZ2UgPSBGQUxTRSwKICAgICAgICAgICAgICAgICAgICAgIHdhcm5pbmcgPSBGQUxTRSkKCiMgZGlzYWJsZSBzY2llbnRpZmljIG5vdGF0aW9uCm9wdGlvbnMoc2NpcGVuID0gOTk5KSAKCmBgYAoKIyBEZXBlbmRlbmNpZXMKCmBgYHtyfQoKbGlicmFyeSh0aWR5cikKbGlicmFyeShkcGx5cikKbGlicmFyeShwdXJycikgCmxpYnJhcnkoc3RyaW5ncikKbGlicmFyeShmb3JjYXRzKQpsaWJyYXJ5KGdncGxvdDIpCmxpYnJhcnkoc2NhbGVzKQpsaWJyYXJ5KHBhdGNod29yaykKbGlicmFyeShrbml0cikKbGlicmFyeShrYWJsZUV4dHJhKQpsaWJyYXJ5KGphbml0b3IpCmxpYnJhcnkoZWZmc2l6ZSkKCmBgYAoKIyBTaW11bGF0aW9uCgpgYGB7cn0KCiMgZnVuY3Rpb25zIGZvciBzaW11bGF0aW9uCmdlbmVyYXRlX2RhdGEgPC0gZnVuY3Rpb24obl9wZXJfY29uZGl0aW9uLAogICAgICAgICAgICAgICAgICAgICAgICAgIG1lYW5fY29udHJvbCwKICAgICAgICAgICAgICAgICAgICAgICAgICBtZWFuX2ludGVydmVudGlvbiwKICAgICAgICAgICAgICAgICAgICAgICAgICBzZCkgewogIAogIGRhdGFfY29udHJvbCA8LSAKICAgIHRpYmJsZShjb25kaXRpb24gPSAiY29udHJvbCIsCiAgICAgICAgICAgc2NvcmUgPSBybm9ybShuID0gbl9wZXJfY29uZGl0aW9uLCBtZWFuID0gbWVhbl9jb250cm9sLCBzZCA9IHNkKSkKICAKICBkYXRhX2ludGVydmVudGlvbiA8LSAKICAgIHRpYmJsZShjb25kaXRpb24gPSAiaW50ZXJ2ZW50aW9uIiwKICAgICAgICAgICBzY29yZSA9IHJub3JtKG4gPSBuX3Blcl9jb25kaXRpb24sIG1lYW4gPSBtZWFuX2ludGVydmVudGlvbiwgc2QgPSBzZCkpCiAgCiAgZGF0YV9jb21iaW5lZCA8LSBiaW5kX3Jvd3MoZGF0YV9jb250cm9sLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGFfaW50ZXJ2ZW50aW9uKSB8PgogICAgbXV0YXRlKGNvbmRpdGlvbiA9IGZjdF9yZWxldmVsKGNvbmRpdGlvbiwgImludGVydmVudGlvbiIsICJjb250cm9sIikpCiAgCiAgcmV0dXJuKGRhdGFfY29tYmluZWQpCn0KCmFuYWx5emUgPC0gZnVuY3Rpb24oZGF0YSkgewoKICByZXNfdF90ZXN0IDwtIHQudGVzdChmb3JtdWxhID0gc2NvcmUgfiBjb25kaXRpb24sIAogICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSBkYXRhLAogICAgICAgICAgICAgICAgICAgICAgIHZhci5lcXVhbCA9IFRSVUUsCiAgICAgICAgICAgICAgICAgICAgICAgYWx0ZXJuYXRpdmUgPSAidHdvLnNpZGVkIikKICAKICByZXNfY29oZW5zX2QgPC0gZWZmc2l6ZTo6Y29oZW4uZChmb3JtdWxhID0gc2NvcmUgfiBjb25kaXRpb24sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YSA9IGRhdGEsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcG9vbGVkID0gVFJVRSkKICAKICByZXMgPC0gdGliYmxlKHAgPSByZXNfdF90ZXN0JHAudmFsdWUsIAogICAgICAgICAgICAgICAgY29oZW5zX2QgPSByZXNfY29oZW5zX2QkZXN0aW1hdGUsCiAgICAgICAgICAgICAgICBjb2hlbnNfZF9jaV9sb3dlciA9IHJlc19jb2hlbnNfZCRjb25mLmludFsxXSwKICAgICAgICAgICAgICAgIGNvaGVuc19kX2NpX3VwcGVyID0gcmVzX2NvaGVuc19kJGNvbmYuaW50WzJdKQoKICByZXR1cm4ocmVzKQp9CgoKIyBzZXQgc2VlZApzZXQuc2VlZCg0MikKCiMgc2ltdWxhdGlvbiBwYXJhbWV0ZXJzCmV4cGVyaW1lbnRfcGFyYW1ldGVycyA8LSBleHBhbmRfZ3JpZCgKICBuX3Blcl9jb25kaXRpb24gPSBzZXEoZnJvbSA9IDEwLCB0byA9IDkwLCBieSA9IDEwKSwKICBtZWFuX2NvbnRyb2wgPSAwLAogIG1lYW5faW50ZXJ2ZW50aW9uID0gMC41LAogIHNkID0gMSwKICBpdGVyYXRpb24gPSAxOjEwMDAKKSAKCiMgcnVuIHNpbXVsYXRpb24Kc2ltdWxhdGlvbiA8LSBleHBlcmltZW50X3BhcmFtZXRlcnMgfD4KICBtdXRhdGUoZ2VuZXJhdGVkX2RhdGEgPSBwbWFwKGxpc3Qobl9wZXJfY29uZGl0aW9uLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWVhbl9jb250cm9sLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtZWFuX2ludGVydmVudGlvbiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2QpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZ2VuZXJhdGVfZGF0YSkpIHw+CiAgbXV0YXRlKHJlc3VsdHMgPSBwbWFwKGxpc3QoZ2VuZXJhdGVkX2RhdGEpLAogICAgICAgICAgICAgICAgICAgICAgICBhbmFseXplKSkKCmBgYAoKIyBDb2hlbidzIGQgYnkgc2FtcGxlIHNpemUKCmBgYHtyIGZpZy5oZWlnaHQ9NSwgZmlnLndpZHRoPTZ9CgojIHdyYW5nbGUKc2ltdWxhdGlvbiB8PgogIHVubmVzdChyZXN1bHRzKSB8PgogICMgcGxvdAogIGdncGxvdChhZXMobl9wZXJfY29uZGl0aW9uKjIsIGNvaGVuc19kKSkgKwogIGdlb21faml0dGVyKGFscGhhID0gMC4yNSkgKwogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gImRvdHRlZCIpICsKICBzY2FsZV94X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX3ByZXR0eShuID0gMTApLAogICAgICAgICAgICAgICAgICAgICBuYW1lID0gIlRvdGFsIE4iKSArCiAgc2NhbGVfeV9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19wcmV0dHkobiA9IDEwKSwKICAgICAgICAgICAgICAgICAgICAgI2xpbWl0cyA9IGMoMCwxKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJDb2hlbidzIGQiKSArCiAgdGhlbWVfbGluZWRyYXcoKSArCiAgc2NhbGVfY29sb3JfdmlyaWRpc19kKGJlZ2luID0gMC4zLCBlbmQgPSAwLjcpICsKICBnZ3RpdGxlKCJQb3B1bGF0aW9uIENvaGVuJ3MgZCA9IDAuNSIpCgpgYGAKCiMjIE9yZGVyZWQgYnkgc3RhdGlzdGljYWwgc2lnbmlmaWNhbmNlCgpMZXQncyBjb2xvciB0aGUgQ29oZW4ncyBkcyBieSB0aGVpciBzdGF0aXN0aWNhbCBzaWduaWZpY2FuY2UuIAoKYGBge3IgZmlnLmhlaWdodD01LCBmaWcud2lkdGg9Nn0KCiMgd3JhbmdsZQpzaW11bGF0aW9uIHw+CiAgdW5uZXN0KHJlc3VsdHMpIHw+CiAgbXV0YXRlKHNpZ25pZmljYW50ID0gcCA8IC4wNSkgfD4KICAjIHBsb3QKICBnZ3Bsb3QoYWVzKG5fcGVyX2NvbmRpdGlvbioyLCBjb2hlbnNfZCwgY29sb3IgPSBzaWduaWZpY2FudCkpICsKICBnZW9tX2ppdHRlcihhbHBoYSA9IDAuMjUpICsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQgPSAwLCBsaW5ldHlwZSA9ICJkb3R0ZWQiKSArCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19wcmV0dHkobiA9IDEwKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJUb3RhbCBOIikgKwogIHNjYWxlX3lfY29udGludW91cyhicmVha3MgPSBicmVha3NfcHJldHR5KG4gPSAxMCksCiAgICAgICAgICAgICAgICAgICAgICNsaW1pdHMgPSBjKDAsMSksCiAgICAgICAgICAgICAgICAgICAgIG5hbWUgPSAiQ29oZW4ncyBkIikgKwogIHRoZW1lX2xpbmVkcmF3KCkgKwogIHNjYWxlX2NvbG9yX3ZpcmlkaXNfZChiZWdpbiA9IDAuMywgZW5kID0gMC43KSArCiAgZ2d0aXRsZSgiUG9wdWxhdGlvbiBDb2hlbidzIGQgPSAwLjUiKQoKYGBgCgojIENvaGVuJ3MgZCBhbmQgaXRzIDk1JSBDSXMgCgpMZXQncyBhZGQgdGhlIDk1JSBDb25maWRlbmNlIEludGVydmFscy4KCiMjIFJhbmRvbWx5IHNlbGVjdCBvbmUgZGF0YXNldCBwZXIgc2FtcGxlIHNpemUKCmBgYHtyfQoKIyB3cmFuZ2xlCnNpbXVsYXRpb24gfD4KICB1bm5lc3QocmVzdWx0cykgfD4KICBtdXRhdGUoc2lnbmlmaWNhbnQgPSBwIDwgLjA1KSB8PgogIGdyb3VwX2J5KG5fcGVyX2NvbmRpdGlvbikgfD4KICBzbGljZV9zYW1wbGUobiA9IDEpIHw+CiAgdW5ncm91cCgpIHw+CiAgIyBwbG90CiAgZ2dwbG90KGFlcyhuX3Blcl9jb25kaXRpb24qMiwgY29oZW5zX2QsIGNvbG9yID0gc2lnbmlmaWNhbnQpKSArCiAgZ2VvbV9wb2ludCgpICsKICBnZW9tX2xpbmVyYW5nZShhZXMoeW1pbiA9IGNvaGVuc19kX2NpX2xvd2VyLCB5bWF4ID0gY29oZW5zX2RfY2lfdXBwZXIpKSArCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0ID0gMCwgbGluZXR5cGUgPSAiZG90dGVkIikgKwogIHNjYWxlX3hfY29udGludW91cyhicmVha3MgPSBicmVha3NfcHJldHR5KG4gPSAxMCksCiAgICAgICAgICAgICAgICAgICAgIG5hbWUgPSAiVG90YWwgTiIpICsKICBzY2FsZV95X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX3ByZXR0eShuID0gMTApLAogICAgICAgICAgICAgICAgICAgICAjbGltaXRzID0gYygwLDEpLAogICAgICAgICAgICAgICAgICAgICBuYW1lID0gIkNvaGVuJ3MgZCIpICsKICB0aGVtZV9saW5lZHJhdygpICsKICBzY2FsZV9jb2xvcl92aXJpZGlzX2QoYmVnaW4gPSAwLjMsIGVuZCA9IDAuNykgKwogIGdndGl0bGUoIlBvcHVsYXRpb24gQ29oZW4ncyBkID0gMC41XG5PbmUgcmFuZG9tbHkgc2VsZWN0ZWQgZGF0YXNldCBwZXIgc2FtcGxlIHNpemUiKQoKYGBgCgojIyBBbGwgZGF0YXNldHMKCmBgYHtyIGZpZy5oZWlnaHQ9MTAsIGZpZy53aWR0aD0xMn0KCiMgd3JhbmdsZQpzaW11bGF0aW9uIHw+CiAgdW5uZXN0KHJlc3VsdHMpIHw+CiAgbXV0YXRlKHNpZ25pZmljYW50ID0gcCA8IC4wNSwKICAgICAgICAgdG90YWxfbiA9IHBhc3RlKCJOID0iLCBuX3Blcl9jb25kaXRpb24qMiksCiAgICAgICAgIHRvdGFsX24gPSBmY3RfcmVvcmRlcih0b3RhbF9uLCBhcy5udW1lcmljKHN0cl9leHRyYWN0KHRvdGFsX24sICJcXGQrIikpKSkgfD4KICAjIHBsb3QKICBnZ3Bsb3QoYWVzKGl0ZXJhdGlvbiwgY29oZW5zX2QsIGNvbG9yID0gc2lnbmlmaWNhbnQpKSArCiAgZ2VvbV9wb2ludChhbHBoYSA9IDAuNSkgKwogIGdlb21fbGluZXJhbmdlKGFlcyh5bWluID0gY29oZW5zX2RfY2lfbG93ZXIsIHltYXggPSBjb2hlbnNfZF9jaV91cHBlciksIGFscGhhID0gMC4yKSArCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0ID0gMCwgbGluZXR5cGUgPSAiZG90dGVkIikgKwogIHNjYWxlX3hfY29udGludW91cyhicmVha3MgPSBicmVha3NfcHJldHR5KG4gPSAxMCksCiAgICAgICAgICAgICAgICAgICAgIG5hbWUgPSAiSXRlcmF0aW9uIikgKwogIHNjYWxlX3lfY29udGludW91cyhicmVha3MgPSBicmVha3NfcHJldHR5KG4gPSAxMCksCiAgICAgICAgICAgICAgICAgICAgICNsaW1pdHMgPSBjKDAsMSksCiAgICAgICAgICAgICAgICAgICAgIG5hbWUgPSAiQ29oZW4ncyBkIikgKwogIHRoZW1lX2xpbmVkcmF3KCkgKwogIHNjYWxlX2NvbG9yX3ZpcmlkaXNfZChiZWdpbiA9IDAuMywgZW5kID0gMC43KSArCiAgZmFjZXRfd3JhcCh+IHRvdGFsX24sIG5jb2wgPSAzLCBzY2FsZXMgPSAiZnJlZV95IikKCmBgYAoKIyMgT3JkZXJlZCBieSBDb2hlbidzIGQKClRoZSBhYm92ZSBwbG90IGlzIGhhcmQgdG8gdW5kZXJzdGFuZC4gTGV0J3Mgb3JkZXIgdGhlIENvaGVuJ3MgZHMgZnJvbSBzbWFsbGVzdCB0byBsYXJnZXN0LiAKCmBgYHtyIGZpZy5oZWlnaHQ9MTAsIGZpZy53aWR0aD0xMn0KCiMgd3JhbmdsZQpzaW11bGF0aW9uIHw+CiAgdW5uZXN0KHJlc3VsdHMpIHw+CiAgbXV0YXRlKHNpZ25pZmljYW50ID0gcCA8IC4wNSwKICAgICAgICAgdG90YWxfbiA9IHBhc3RlKCJOID0iLCBuX3Blcl9jb25kaXRpb24qMiksCiAgICAgICAgIHRvdGFsX24gPSBmY3RfcmVvcmRlcih0b3RhbF9uLCBhcy5udW1lcmljKHN0cl9leHRyYWN0KHRvdGFsX24sICJcXGQrIikpKSkgfD4KICBhcnJhbmdlKG5fcGVyX2NvbmRpdGlvbiwgY29oZW5zX2QpIHw+CiAgZ3JvdXBfYnkobl9wZXJfY29uZGl0aW9uKSB8PgogIG11dGF0ZShyYW5rID0gcm93X251bWJlcigpKSB8PgogIHVuZ3JvdXAoKSB8PgogICMgcGxvdAogIGdncGxvdChhZXMocmFuaywgY29oZW5zX2QsIGNvbG9yID0gc2lnbmlmaWNhbnQpKSArCiAgZ2VvbV9wb2ludCgpICsKICBnZW9tX2xpbmVyYW5nZShhZXMoeW1pbiA9IGNvaGVuc19kX2NpX2xvd2VyLCB5bWF4ID0gY29oZW5zX2RfY2lfdXBwZXIpLCBhbHBoYSA9IDAuMikgKwogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gImRvdHRlZCIpICsKICBzY2FsZV94X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX3ByZXR0eShuID0gMTApLAogICAgICAgICAgICAgICAgICAgICBuYW1lID0gIlJhbmtlZCBpdGVyYXRpb24iKSArCiAgc2NhbGVfeV9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19wcmV0dHkobiA9IDEwKSwKICAgICAgICAgICAgICAgICAgICAgI2xpbWl0cyA9IGMoMCwxKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJDb2hlbidzIGQiKSArCiAgdGhlbWVfbGluZWRyYXcoKSArCiAgc2NhbGVfY29sb3JfdmlyaWRpc19kKGJlZ2luID0gMC4zLCBlbmQgPSAwLjcpICsKICBmYWNldF93cmFwKH4gdG90YWxfbiwgbmNvbCA9IDMpCgpgYGAKCi0gTm90aWNlIHRoZSByZWxhdGlvbnNoaXAgYmV0d2VlbiA5NSUgQ0kgYW5kIHAgdmFsdWUgc2lnbmlmaWNhbmNlLgotIEEgY2VydGFpbiBwZXJjZW50YWdlIG9mIENvaGVuJ3MgZHMgYXJlIGdyZWVuIHZzLiBibHVlLiBXaGF0IGlzIHRoaXMgcGVyY2VudGFnZSBhbHNvIGNhbGxlZD8gV2hhdCBzdGF0aXN0aWNhbCBwcm9wZXJ0eT8KCiMgTWVhbiBDb2hlbidzIGQgYnkgc2FtcGxlIHNpemUKCk5vdyBsZXQncyBhdmVyYWdlIG92ZXIgdGhlIENvaGVuJ3MgZHMgaW4gZWFjaCBjb25kaXRpb24gdG8gZmluZCB0aGUgbWVhbiBDb2hlbidzIGQgYW5kIGl0cyBtZWFuIDk1JSBDSXMuCgpgYGB7ciBmaWcuaGVpZ2h0PTIuNSwgZmlnLndpZHRoPTZ9CgojIHdyYW5nbGUKc2ltdWxhdGlvbl9zdW1tYXJ5IDwtIHNpbXVsYXRpb24gfD4KICAjIHVubmVzdCByZXN1bHRzCiAgdW5uZXN0KHJlc3VsdHMpIHw+CiAgZ3JvdXBfYnkobl9wZXJfY29uZGl0aW9uKSB8PgogIHN1bW1hcml6ZShwcm9wb3J0aW9uX3NpZ25pZmljYW50ID0gbWVhbihwIDwgLjA1KSwgCiAgICAgICAgICAgIG1lYW5fY29oZW5zX2QgPSBtZWFuKGNvaGVuc19kKSwKICAgICAgICAgICAgbWVhbl9jb2hlbnNfZF9jaV9sb3dlciA9IG1lYW4oY29oZW5zX2RfY2lfbG93ZXIpLAogICAgICAgICAgICBtZWFuX2NvaGVuc19kX2NpX3VwcGVyID0gbWVhbihjb2hlbnNfZF9jaV91cHBlcikpIHw+CiAgbXV0YXRlKGNlbnRlcmVkX21lYW5fY29oZW5zX2RfY2lfbG93ZXIgPSBtZWFuX2NvaGVuc19kX2NpX2xvd2VyIC0gbWVhbl9jb2hlbnNfZCwKICAgICAgICAgY2VudGVyZWRfbWVhbl9jb2hlbnNfZF9jaV91cHBlciA9IG1lYW5fY29oZW5zX2RfY2lfdXBwZXIgLSBtZWFuX2NvaGVuc19kKQoKIyBwbG90IHJlc3VsdHMKcDEgPC0gZ2dwbG90KHNpbXVsYXRpb25fc3VtbWFyeSwgYWVzKG5fcGVyX2NvbmRpdGlvbioyLCBtZWFuX2NvaGVuc19kKSkgKwogIGdlb21fcG9pbnQoKSArCiAgZ2VvbV9saW5lcmFuZ2UoYWVzKHltaW4gPSBtZWFuX2NvaGVuc19kX2NpX2xvd2VyLCB5bWF4ID0gbWVhbl9jb2hlbnNfZF9jaV91cHBlcikpICsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQgPSAwLCBsaW5ldHlwZSA9ICJkb3R0ZWQiKSArCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19wcmV0dHkobiA9IDEwKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJUb3RhbCBzYW1wbGUgc2l6ZSIpICsKICBzY2FsZV95X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX3ByZXR0eShuID0gNSksCiAgICAgICAgICAgICAgICAgICAgICNsaW1pdHMgPSBjKDAsMSksCiAgICAgICAgICAgICAgICAgICAgIG5hbWUgPSAiTWVhbiBDb2hlbidzIGRcbihhbmQgbWVhbiA5NSUgQ0lzKSIpICsKICB0aGVtZV9saW5lZHJhdygpICsKICBnZ3RpdGxlKCJQb3B1bGF0aW9uIENvaGVuJ3MgZCA9IDAuNSIpCgpwMQoKYGBgCgojIE1lYW4gQ29oZW4ncyBkIGFuZCBwb3dlciBieSBzYW1wbGUgc2l6ZQoKYGBge3IgZmlnLmhlaWdodD01LCBmaWcud2lkdGg9Nn0KCnAyIDwtIGdncGxvdChzaW11bGF0aW9uX3N1bW1hcnksIGFlcyhuX3Blcl9jb25kaXRpb24qMiwgcHJvcG9ydGlvbl9zaWduaWZpY2FudCkpICsKICBnZW9tX3BvaW50KCkgKwogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAuMDUsIGxpbmV0eXBlID0gImRvdHRlZCIpICsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQgPSAwLjgwLCBsaW5ldHlwZSA9ICJkb3R0ZWQiKSArCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcyA9IGJyZWFrc19wcmV0dHkobiA9IDEwKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJUb3RhbCBzYW1wbGUgc2l6ZSIpICsKICBzY2FsZV95X2NvbnRpbnVvdXMoYnJlYWtzID0gYnJlYWtzX3ByZXR0eShuID0gNSksCiAgICAgICAgICAgICAgICAgICAgIGxpbWl0cyA9IGMoMCwxKSwKICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJQcm9wb3J0aW9uIG9mIHNpZ25pZmljYW50XG5wLXZhbHVlcyIpICsKICB0aGVtZV9saW5lZHJhdygpIAoKcDEgKyBwMiArIHBsb3RfbGF5b3V0KG5jb2wgPSAxKQoKYGBgCgojIFNlc3Npb24gaW5mbwoKYGBge3J9CgpzZXNzaW9uSW5mbygpCgpgYGAKCgo=